package com.alex.sitegrabber;

import java.io.File;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.io.FileUtils;
import org.apache.http.HttpHost;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.protocol.ExecutionContext;

import com.alex.utils.http.HttpClientFactory;
import com.alex.utils.http.HttpResult;
import com.alex.utils.http.HttpSession;
import com.alex.utils.http.RequestBuilderFactory;

public class SiteGrabber2 {

	private static final String SITE_FILES_DIR = "files/";

	private static final Pattern IMG_PATTERN = Pattern.compile("<img.+?src=[\"']([^\"']+)[\"'][^>]*>", Pattern.CASE_INSENSITIVE);
	
	private static final Pattern STYLE_LINK_PATTERN = Pattern.compile("<link.+?href=[\"']([^\"']+)[\"'][^>]*>", Pattern.CASE_INSENSITIVE);
	
	private static final Pattern ANCHOR_PATTERN = Pattern.compile("<a.+?href=[\"']([^\"']+)[\"'][^>]*>", Pattern.CASE_INSENSITIVE);
	
	private static final Pattern STYLE_URL_PATTERN = Pattern.compile("url\\([\"']?([^\"'()]+)[\"']?\\)", Pattern.CASE_INSENSITIVE);
	
	private RequestBuilderFactory requestBuilderFactory;

	private URL url;
	
	private String host;
	
	private File saveTo;
	
	private Map<String, String> resources = new HashMap<String, String>();
	
	private Set<String> usedNames = new HashSet<String>();
	
	private SiteGrabber2(String url, String saveTo) throws MalformedURLException {
		this.url = new URL(url);
		this.saveTo = new File(saveTo);
		this.saveTo.mkdirs();
		this.host = this.url.getHost();
		
		HttpClientFactory httpClientFactory = new HttpClientFactory();
		httpClientFactory.setThreadSafe(false);
		requestBuilderFactory = new RequestBuilderFactory(httpClientFactory.create());
	}
	
	public static void grabSite(String url, String saveTo) throws IOException {
		new SiteGrabber2(url, saveTo).grabSite();
	}

	private void grabSite() throws IOException {
		final Map<String, String> linkResolveMap = new HashMap<String, String>();
		final LinkedList<URL> queue = new LinkedList<URL>();
		final Set<URL> urlSet = new HashSet<URL>();
		
		queue.add(url);
		urlSet.add(url);
		while (!queue.isEmpty()) {
			final URL url = queue.removeFirst();
			String stringUrl = url.toString();
			
			System.out.println("Processing URL: " + stringUrl + ", left " + queue.size());
			
			HttpResult httpResult = null;
			try {
				HttpSession session = HttpSession.createSession();
				httpResult = requestBuilderFactory.get(url.toString()).session(session).getResult();
				if (httpResult.getResponse().getStatusLine().getStatusCode() != 200) {
					throw new IOException("Error downloading " + url);
				}
				String page = httpResult.getResponseBodyAsString();

				HttpUriRequest targetRequest = (HttpUriRequest) session.getHttpContext().getAttribute(ExecutionContext.HTTP_REQUEST);
				HttpHost targetHost = (HttpHost) session.getHttpContext().getAttribute(ExecutionContext.HTTP_TARGET_HOST);
				String targetUrl = targetHost.toURI() + targetRequest.getURI();
				
				if (!targetHost.getHostName().equalsIgnoreCase(url.getHost())) {
					System.out.println("Target host does not match: " + targetUrl);
					continue;
				}
				
				linkResolveMap.put(stringUrl, targetUrl);
				
				RegexProcessor.process(page, ANCHOR_PATTERN, new ReplacementHandler() {
					@Override
					public String handleReplacement(String anchor) throws IOException {
						URL link = new URL(url, anchor);
						if (link.getHost().equalsIgnoreCase(host) && !urlSet.contains(link)) {
							queue.add(link);
							urlSet.add(link);
						}
						return anchor;
					}
				});
				
			} finally {
				if (httpResult != null) {
					httpResult.close();
				}
			}
		}
		
//		String page = getURLString(url.toString());
//		page = processLinks(page);
//		page = processImages(page);
//		page = processStyleUrls(page, SITE_FILES_DIR);
//		FileUtils.write(new File(saveTo, "index.html"), page);
	}

	private String processStyleLinks(String page) throws IOException {
		int end = 0;
		StringBuilder sb = new StringBuilder();
		Matcher matcher = STYLE_LINK_PATTERN.matcher(page);
		while (matcher.find()) {
			sb.append(page.substring(end, matcher.start(1)));
			end = matcher.end(1);
			String link = matcher.group(1);
			sb.append(SITE_FILES_DIR + downloadResource(link, false, new ResourceProcessor() {
				@Override
				public String processResourse(String resource) throws IOException {
					return processStyleUrls(resource, "");
				}
			}));
		}
		sb.append(page.substring(end));
		return sb.toString();
	}

	private String processImages(String page) throws IOException {
		int end = 0;
		StringBuilder sb = new StringBuilder();
		Matcher matcher = IMG_PATTERN.matcher(page);
		while (matcher.find()) {
			sb.append(page.substring(end, matcher.start(1)));
			end = matcher.end(1);
			String img = matcher.group(1);
			sb.append(SITE_FILES_DIR + downloadResource(img, true));
		}
		sb.append(page.substring(end));
		return sb.toString();
	}
	
	private String processStyleUrls(String page, String path) throws IOException {
		int end = 0;
		StringBuilder sb = new StringBuilder();
		Matcher matcher = STYLE_URL_PATTERN.matcher(page);
		while (matcher.find()) {
			sb.append(page.substring(end, matcher.start(1)));
			end = matcher.end(1);
			String url = matcher.group(1);
			sb.append(path + downloadResource(url, true));
		}
		sb.append(page.substring(end));
		return sb.toString();
	}

	private String downloadResource(String resource, boolean binary) throws IOException {
		return downloadResource(resource, binary, null);
	}

	private String downloadResource(String resource, boolean binary, ResourceProcessor processor) throws IOException {
		String downloadedResource = resources.get(resource);
		if (downloadedResource != null) {
			return downloadedResource;
		}
		
		System.out.println(resource);
		
		URL resourceURL = new URL(url, resource);
		String file = getFileName(resource);
		
		HttpResult httpResult = null;
		try {
			httpResult = requestBuilderFactory.get(resourceURL.toString()).getResult();
			if (!binary) {
				String data = httpResult.getResponseBodyAsString();
				if (processor != null) {
					data = processor.processResourse(data);
				}
				FileUtils.write(new File(saveTo, SITE_FILES_DIR + file), data);
			} else {
				byte[] data = httpResult.getResponseBodyAsByteArray();
				FileUtils.writeByteArrayToFile(new File(saveTo, SITE_FILES_DIR + file), data);
			}
			resources.put(resource, file);
			return file;
		} finally {
			if (httpResult != null) {
				httpResult.close();
			}
		}
	}

	private String getFileName(String resource) {
		String name;
		int p = resource.lastIndexOf("/");
		if (p < 0) {
			name = resource;
		} else {
			name = resource.substring(p + 1);
		}
		int i = 1;
		if (usedNames.contains(name)) {
			String nameWithNumber = null;
			do {
				nameWithNumber = name + i;
			} while (usedNames.contains(nameWithNumber));
			usedNames.add(nameWithNumber);
			return nameWithNumber;
		} else {
			usedNames.add(name);
			return name;
		}
	}

	private String downloadPage(String url) throws IOException {
		HttpResult httpResult = null;
		try {
			httpResult = requestBuilderFactory.get(url).getResult();
			if (httpResult.getResponse().getStatusLine().getStatusCode() != 200) {
				throw new IOException("Error downloading " + url);
			}
			return httpResult.getResponseBodyAsString();
		} finally {
			if (httpResult != null) {
				httpResult.close();
			}
		}
	}

	public static void main(String[] args) throws IOException {
		new SiteGrabber2("http://www.generalmovs.com/categories.shtml", "/tmp/site").grabSite();
	}
}
